<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css">

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"yoursite.com","root":"/","scheme":"Gemini","version":"7.8.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":true,"show_result":true,"style":"flat"},"back2top":{"enable":true,"sidebar":true,"scrollpercent":true},"bookmark":{"enable":true,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},"path":"search.xml"};
  </script>

  <meta name="description" content="python爬虫 （2020.2.25-2.28）基础知识爬虫  爬虫就是模拟客户端（浏览器）发送网络请求，获取响应，按照规制提取数据的查询  数据处理  直接呈现：展现在网页或   者app中 进行分析：从数据中寻找一些规律  浏览器的请求 HTTP和HTTPS HTTP协议">
<meta property="og:type" content="article">
<meta property="og:title" content="Python爬虫">
<meta property="og:url" content="http://yoursite.com/2020/02/26/%E5%90%8E%E7%AB%AF/python/python%E7%88%AC%E8%99%AB/index.html">
<meta property="og:site_name" content="heweiliang blog">
<meta property="og:description" content="python爬虫 （2020.2.25-2.28）基础知识爬虫  爬虫就是模拟客户端（浏览器）发送网络请求，获取响应，按照规制提取数据的查询  数据处理  直接呈现：展现在网页或   者app中 进行分析：从数据中寻找一些规律  浏览器的请求 HTTP和HTTPS HTTP协议">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2020-02-26T07:44:28.352Z">
<meta property="article:modified_time" content="2020-06-26T08:19:49.114Z">
<meta property="article:author" content="heweiliang">
<meta property="article:tag" content="Python">
<meta property="article:tag" content="爬虫">
<meta name="twitter:card" content="summary">

<link rel="canonical" href="http://yoursite.com/2020/02/26/%E5%90%8E%E7%AB%AF/python/python%E7%88%AC%E8%99%AB/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh-CN'
  };
</script>

  <title>Python爬虫 | heweiliang blog</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

<link rel="alternate" href="/atom.xml" title="heweiliang blog" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">heweiliang blog</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
        <i class="fa fa-search fa-fw fa-lg"></i>
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-fw fa-home"></i>首页</a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="fa fa-fw fa-user"></i>关于</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-fw fa-tags"></i>标签</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-fw fa-th"></i>分类</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-fw fa-archive"></i>归档</a>

  </li>
        <li class="menu-item menu-item-schedule">

    <a href="/schedule/" rel="section"><i class="fa fa-fw fa-calendar"></i>日程表</a>

  </li>
        <li class="menu-item menu-item-sitemap">

    <a href="/sitemap.xml" rel="section"><i class="fa fa-fw fa-sitemap"></i>站点地图</a>

  </li>
      <li class="menu-item menu-item-search">
        <a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>搜索
        </a>
      </li>
  </ul>
</nav>



  <div class="search-pop-overlay">
    <div class="popup search-popup">
        <div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container">
    <input autocomplete="off" autocapitalize="off"
           placeholder="搜索..." spellcheck="false"
           type="search" class="search-input">
  </div>
  <span class="popup-btn-close">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div id="search-result">
  <div id="no-result">
    <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
  </div>
</div>

    </div>
  </div>

</div>
    </header>

    
  <div class="reading-progress-bar"></div>
  <a role="button" class="book-mark-link book-mark-link-fixed"></a>

  <a href="https://github.com/yourname" class="github-corner" title="Follow me on GitHub" aria-label="Follow me on GitHub" rel="noopener" target="_blank"><svg width="80" height="80" viewBox="0 0 250 250" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2020/02/26/%E5%90%8E%E7%AB%AF/python/python%E7%88%AC%E8%99%AB/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.jpg">
      <meta itemprop="name" content="heweiliang">
      <meta itemprop="description" content="heweiliang blog">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="heweiliang blog">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          Python爬虫<a href="https://github.com/user-name/repo-name/tree/branch-name/subdirectory-name_posts/%E5%90%8E%E7%AB%AF/python/python%E7%88%AC%E8%99%AB.md" class="post-edit-link" title="编辑" rel="noopener" target="_blank"><i class="fa fa-pencil"></i></a>
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              <time title="创建时间：2020-02-26 15:44:28" itemprop="dateCreated datePublished" datetime="2020-02-26T15:44:28+08:00">2020-02-26</time>
            </span>
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2020-06-26 16:19:49" itemprop="dateModified" datetime="2020-06-26T16:19:49+08:00">2020-06-26</time>
              </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Python/" itemprop="url" rel="index"><span itemprop="name">Python</span></a>
                </span>
            </span>

          
            <span class="post-meta-item" title="阅读次数" id="busuanzi_container_page_pv" style="display: none;">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span id="busuanzi_value_page_pv"></span>
            </span><br>
            <span class="post-meta-item" title="本文字数">
              <span class="post-meta-item-icon">
                <i class="fa fa-file-word-o"></i>
              </span>
                <span class="post-meta-item-text">本文字数：</span>
              <span>17k</span>
            </span>
            <span class="post-meta-item" title="阅读时长">
              <span class="post-meta-item-icon">
                <i class="fa fa-clock-o"></i>
              </span>
                <span class="post-meta-item-text">阅读时长 &asymp;</span>
              <span>16 分钟</span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <h2 id="python爬虫-（2020-2-25-2-28）"><a href="#python爬虫-（2020-2-25-2-28）" class="headerlink" title="python爬虫 （2020.2.25-2.28）"></a>python爬虫 （2020.2.25-2.28）</h2><h4 id="基础知识"><a href="#基础知识" class="headerlink" title="基础知识"></a>基础知识</h4><p><strong>爬虫</strong></p>
<ul>
<li>爬虫就是模拟客户端（浏览器）发送网络请求，获取响应，按照规制提取数据的查询</li>
</ul>
<p><strong>数据处理</strong></p>
<ul>
<li>直接呈现：展现在网页或   者app中</li>
<li>进行分析：从数据中寻找一些规律</li>
</ul>
<p><strong>浏览器的请求</strong></p>
<p><strong>HTTP和HTTPS</strong></p>
<p><strong>HTTP协议</strong></p>
<ol>
<li>概念</li>
</ol>
<ul>
<li>Hypertext Transer Protocol,超文本传输协议</li>
<li>HTTP是一个基于“请求与响应模式的、无状态的应用层协议”</li>
<li>HTTP协议采用URL作为定位网络资源的标识</li>
</ul>
<ol start="2">
<li>例子：url = <a href="http://host" target="_blank" rel="noopener">http://host</a>: post /path</li>
</ol>
<p>​        http/https 请求协议</p>
<p>​        host:internet主机域名或IP地址<br>​        port:端口号，缺省端口为80<br>​        path:请求资源的路径</p>
<p>​        url = 请求协议+网站域名+资源路径+参数</p>
<ol start="3">
<li>请求url地址</li>
</ol>
<ul>
<li><p>浏览器请求的url地址（网页）</p>
<p>当前url对应的响应+js+css+图片+视频 ======》elements中的内容</p>
</li>
<li><p>爬虫请求的url地址（网页源码和相应的资源）</p>
<p>当前url对应的响应</p>
</li>
<li><p>element的内容和爬虫获取到的url地址的响应不同，爬虫中需要以当前url地址对应的响应为准提取数据</p>
</li>
</ul>
<ol start="4">
<li><p>资源的操作<br><strong>GET</strong>获取</p>
<pre><code>请求获取URL位置的资源</code></pre><p><strong>HEAD头部信息</strong></p>
<pre><code>获取该资源的头部信息</code></pre><p><strong>POST</strong></p>
<pre><code>向URL位置的资源附加新的数据</code></pre><p><strong>PUT</strong></p>
<pre><code>向URL位置存储一个资源，覆盖原URL位置的资源</code></pre><p><strong>PATCH更新</strong></p>
<pre><code>更新URL位置的资源
节省网络带宽</code></pre><p><strong>DELETE删除</strong></p>
<pre><code>删除URL位置存储的资源</code></pre></li>
<li><p>HTTP协议之请求</p>
</li>
</ol>
<ul>
<li><p>请求行</p>
</li>
<li><p>请求头</p>
</li>
</ul>
<p>​    User-Agent:用户代理 对方浏览器能通过user-agent知道当前请求对方资源的是什么浏览器</p>
<p>​    Cookie:用来存储用户信息的，每次请求会被携带上发送给对方的浏览器</p>
<p>​    要获取登录才能访问的页面</p>
<p>​    对方的服务器会通过Cookie来判断我们是是一个爬虫</p>
<ul>
<li>请求体（post）</li>
</ul>
<p>​    get请求没有请求体，post请求有请求体，</p>
<p>​    get请求把数据放在url地址中，post请求常用于登录注册，传输大文本的时候</p>
<ol start="6">
<li>HTTP协议之响应</li>
</ol>
<ul>
<li><p>响应头</p>
<p> set-Cookie:对方服务器通过该字段设置cookie到本地</p>
</li>
<li><p>响应体</p>
<p>  url地址对应的响应   </p>
</li>
</ul>
<p><strong>HTTPS协议</strong> HTTP+SSL(安全套接字层)</p>
<p>传输之前先加密，之后解密获取内容</p>
<p>超文本加密协议</p>
<p>HTTP与HTTPS的区别</p>
<p>HTTP 以明文形式传输（不安全）、效率高、不安全</p>
<p>HTTPS SSL的方式加密，效率低、不安全</p>
<h4 id="Requests-Beautiful-Soup-robots-txt-初级"><a href="#Requests-Beautiful-Soup-robots-txt-初级" class="headerlink" title="Requests + Beautiful Soup + robots.txt(初级)"></a>Requests + Beautiful Soup + robots.txt(初级)</h4><h4 id="Requests-爬取HTML页面自动网络请求提交"><a href="#Requests-爬取HTML页面自动网络请求提交" class="headerlink" title="Requests    爬取HTML页面自动网络请求提交"></a>Requests    爬取HTML页面自动网络请求提交</h4><h6 id="概念"><a href="#概念" class="headerlink" title="概念"></a>概念</h6><ul>
<li><p>自动爬取HTML页面自动网络请求提交</p>
</li>
<li><p>小规模、数据量小、爬取速度不敏感</p>
</li>
<li><p>Requests 是用<strong>Python</strong>语言编写，基于 urllib，采用 Apache2 Licensed 开源协议的 HTTP 库。</p>
</li>
<li><p>它比 urllib 更加方便，可以节约我们大量的工作，完全满足 HTTP 测试需求。</p>
</li>
<li><p>Requests 的哲学是以 PEP 20 的习语为中心开发的，所以它比 urllib 更加 Pythoner。更重要的一点是它支持 Python3 哦！</p>
</li>
</ul>
<h6 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h6><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install requests</span><br></pre></td></tr></table></figure>

<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">$ git clone git:&#x2F;&#x2F;github.com&#x2F;kennethreitz&#x2F;requests.git</span><br><span class="line">$ cd requests</span><br><span class="line">$ python setup.py install</span><br></pre></td></tr></table></figure>



<h6 id="requests方法"><a href="#requests方法" class="headerlink" title="requests方法"></a>requests方法</h6><p><strong>requests()</strong></p>
<p><strong>get(url)</strong></p>
<ul>
<li><p>获取HTML网页的主要方法，HTTP的GET</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line">url = <span class="string">"http://baidu.com"</span></span><br><span class="line">response = requests.get(url)</span><br><span class="line">print(response.status_code)</span><br><span class="line">print(response)</span><br><span class="line">print(response.text)</span><br></pre></td></tr></table></figure>

</li>
</ul>
<p><strong>head()</strong></p>
<ul>
<li>获取HTML网页头信息的方法，HTTP的head</li>
</ul>
<p><strong>post(url,data={请求体的字典},headers=headers)</strong></p>
<ul>
<li>向网页提交post请求，对应于HTTP的post</li>
</ul>
<p><strong>put()</strong></p>
<ul>
<li>向网页提交put请求，对应于HTTP的put</li>
</ul>
<p><strong>patch()</strong></p>
<ul>
<li>向网页提交局部修改请求，对应于HTTP的patch</li>
</ul>
<p><strong>delete()</strong></p>
<ul>
<li>向HTML页面提交删除请求，对应于HTTP的delete</li>
</ul>
<p><strong>url</strong></p>
<p>返回网页的地址</p>
<p><strong>status_code</strong></p>
<ul>
<li>状态码</li>
</ul>
<h6 id="response对象的属性"><a href="#response对象的属性" class="headerlink" title="response对象的属性"></a>response对象的属性</h6><p>出现乱码：解码的使用方法</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">response.encoding=<span class="string">"utf-8"</span></span><br><span class="line">response.text</span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 把响应的二进制字节转化为str类型</span></span><br><span class="line">response.content.decode()</span><br><span class="line">response.content.decode(<span class="string">"gbk"</span>)</span><br></pre></td></tr></table></figure>

<h6 id="发送带header的请求-post-get"><a href="#发送带header的请求-post-get" class="headerlink" title="发送带header的请求 post/get"></a>发送带header的请求 post/get</h6><p>headers</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 复制内容位置Google控制台—&gt;NetWork-&gt;Name-&gt;Request Headers</span></span><br><span class="line"><span class="comment">#手机headers</span></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">"user-agent"</span>:</span><br><span class="line">    <span class="string">"Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 		(KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"</span>,</span><br><span class="line">    <span class="string">"referer"</span>:<span class="string">"https://fanyi.baidu.com/"</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">#电脑header2</span></span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">"User-Agent"</span>: </span><br><span class="line">    <span class="string">"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) 	   Chrome/80.0.3987.106 Safari/537.36"</span></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">response = requests.post(url,data=data_string,headers=headers)</span><br></pre></td></tr></table></figure>

<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">response &#x3D; requests.get(url,headers&#x3D;headers)</span><br></pre></td></tr></table></figure>

<p>response.request.url 发送请求的url地址</p>
<p>response.url     response响应的url地址</p>
<p>respone.request.headers 请求头</p>
<p>response.headers 响应请求</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> retrying <span class="keyword">import</span> retry</span><br><span class="line"><span class="meta">@retry(stop_max_attempt_number=3)</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">fun</span><span class="params">()</span>:</span></span><br><span class="line">    print(<span class="string">"this is func"</span>)</span><br><span class="line">    reise ValueError(<span class="string">"this is test error"</span>)</span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line">url = <span class="string">"https://www.baidu.com/"</span></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">"User-Agent"</span>:</span><br><span class="line">    <span class="string">"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) 	   Chrome/80.0.3987.106 Safari/537.36"</span></span><br><span class="line">&#125;</span><br><span class="line">response = requests.get(url,headers=headers)</span><br><span class="line"><span class="comment"># print(response.content.decode())</span></span><br><span class="line">print(response.request.headers)</span><br><span class="line">print(response.headers)</span><br><span class="line">print(response.url)</span><br><span class="line">print(response.request.url)</span><br><span class="line"><span class="comment"># print(response.text)</span></span><br><span class="line"></span><br><span class="line">&#123;<span class="string">'User-Agent'</span>: <span class="string">'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \t   Chrome/80.0.3987.106 Safari/537.36'</span>, <span class="string">'Accept-Encoding'</span>: <span class="string">'gzip, deflate'</span>, <span class="string">'Accept'</span>: <span class="string">'*/*'</span>, <span class="string">'Connection'</span>: <span class="string">'keep-alive'</span>&#125;</span><br><span class="line">&#123;<span class="string">'Bdpagetype'</span>: <span class="string">'1'</span>.....&#125;</span><br><span class="line">https://www.baidu.com/</span><br><span class="line">https://www.baidu.com/</span><br></pre></td></tr></table></figure>

<p>百度翻译例子 </p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line">url = <span class="string">"https://fanyi.baidu.com/basetrans"</span></span><br><span class="line">data_string = &#123;</span><br><span class="line">   <span class="string">"query"</span>:<span class="string">"百度翻译"</span>,</span><br><span class="line">   <span class="string">"from"</span>:<span class="string">"zh"</span>,</span><br><span class="line">   <span class="string">"to"</span>:<span class="string">"en"</span>&#125;</span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">"user-agent"</span>:</span><br><span class="line">    <span class="string">"Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"</span>,</span><br><span class="line">    <span class="comment"># "referer":"https://fanyi.baidu.com/"</span></span><br><span class="line">&#125;</span><br><span class="line">response = requests.post(url,data=data_string,headers=headers)</span><br><span class="line">print(response)</span><br><span class="line">print(response.content.decode())</span><br></pre></td></tr></table></figure>

<p>获取完整的源代码</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line">url = <span class="string">"https://www.baidu.com/"</span></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">"User-Agent"</span>:</span><br><span class="line">    <span class="string">"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) 	   Chrome/80.0.3987.106 Safari/537.36"</span></span><br><span class="line">&#125;</span><br><span class="line">response = requests.get(url,headers=headers)</span><br><span class="line"><span class="comment"># print(response.content.decode())</span></span><br><span class="line">print(response.text)</span><br></pre></td></tr></table></figure>



<h6 id="搜索引擎关键词提交接口"><a href="#搜索引擎关键词提交接口" class="headerlink" title="搜索引擎关键词提交接口"></a>搜索引擎关键词提交接口</h6><ol>
<li>百度</li>
</ol>
<ul>
<li><a href="https://www.baidu.com/s?wd=keyword" target="_blank" rel="noopener">https://www.baidu.com/s?wd=keyword</a></li>
</ul>
<ol start="2">
<li>360</li>
</ol>
<ul>
<li><a href="https://www.so.com/s?s?q=keyword" target="_blank" rel="noopener">https://www.so.com/s?s?q=keyword</a></li>
</ul>
<ol start="3">
<li>实例</li>
</ol>
<h6 id="Retrying模块（解决超时请求问题的库）"><a href="#Retrying模块（解决超时请求问题的库）" class="headerlink" title="Retrying模块（解决超时请求问题的库）"></a>Retrying模块（解决超时请求问题的库）</h6><ol>
<li>超时参数</li>
</ol>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">response = requests.get(url,headers=headers,timeout=<span class="number">3</span>)   <span class="number">3</span>秒内返回响应，否则会报错</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>安装</li>
</ol>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install retrying</span><br></pre></td></tr></table></figure>

<ol start="3">
<li>模板</li>
</ol>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> retrying <span class="keyword">import</span> retry</span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">"User-Agent"</span>:</span><br><span class="line">    <span class="string">"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) 	   Chrome/80.0.3987.106 Safari/537.36"</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="meta">@retry(stop_max_attempt_number=3)</span></span><br><span class="line"><span class="comment"># 让被函数装饰的函数反复执行三次，中间有次正常则退出</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">_parse_url</span><span class="params">(url)</span>:</span></span><br><span class="line">    print(<span class="string">"*"</span>*<span class="number">10</span>)</span><br><span class="line">    response = requests.get(url,headers=headers,timeout=<span class="number">5</span>)</span><br><span class="line">    <span class="keyword">return</span> response.content.decode()</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">parse_url</span><span class="params">(url)</span>:</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        html_str = _parse_url(url)</span><br><span class="line">    <span class="keyword">except</span>:</span><br><span class="line">        html_str = <span class="literal">None</span></span><br><span class="line">    <span class="keyword">return</span>  html_str</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    <span class="comment"># 正确的url地址</span></span><br><span class="line">    url = <span class="string">"http://www.baidu.com"</span></span><br><span class="line">    <span class="comment"># 错误的url地址</span></span><br><span class="line">    url2 = <span class="string">"www.baidu.com"</span></span><br><span class="line">    print(parse_url(url)[:<span class="number">100</span>])</span><br><span class="line">    print(parse_url(url2))</span><br></pre></td></tr></table></figure>



<h6 id="Cookie的相关请求"><a href="#Cookie的相关请求" class="headerlink" title="Cookie的相关请求"></a>Cookie的相关请求</h6><ol>
<li><p>直接携带cookie请求url地址</p>
<ul>
<li>cookie放在headers中</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">headers&#x3D;&#123;</span><br><span class="line">	&quot;Uset-Ageng&quot;:&quot; &quot;,</span><br><span class="line">	&quot;Cookie&quot;:&quot; &quot;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<ul>
<li>cookie字典传输给cookies参数</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">requests.get(url,cookies=cookie_dict)</span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line">url = <span class="string">"https://www.bilibili.com/"</span></span><br><span class="line">headers = &#123;<span class="string">"User-Agent"</span>:</span><br><span class="line">    <span class="string">"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, 		like Gecko) 	   Chrome/80.0.3987.106 Safari/537.36"</span>,</span><br><span class="line">    <span class="comment">#第一种方法 headers</span></span><br><span class="line">     <span class="string">"Cookie"</span>: <span class="string">"_uuid=D60C295E-EEF8-A0E4-B466C58C6D88E65A82945infoc; "</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">#第二种方法 字典</span></span><br><span class="line">cookie_dict = &#123; </span><br><span class="line">    <span class="comment"># _uuid=D60C295E-EEF8-A0E4-B466C58C6D88E65A82945infoc;</span></span><br><span class="line">	i.split(<span class="string">"="</span>)[<span class="number">0</span>]:i.split(<span class="string">"="</span>)[<span class="number">-1</span>] <span class="keyword">for</span> i <span class="keyword">in</span> cookie.split(<span class="string">"; "</span>)</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">response = requests.get(url,headers=headers)</span><br><span class="line"><span class="comment"># response = requests.get(url,headers=headers,cookies=cookie_dict)</span></span><br><span class="line"><span class="keyword">with</span> open(<span class="string">"E01.html"</span>,<span class="string">"w"</span>,encoding=<span class="string">"utf-8"</span>) <span class="keyword">as</span> f: </span><br><span class="line">    f.write(response.content.decode())</span><br></pre></td></tr></table></figure>

<p>2.先发送post请求，获取cookie,带上cookie请求登录后的页面</p>
<p> Seesion参数</p>
<p> 方法</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#session 具有的方法和requests一样</span></span><br><span class="line">seesion = requests.session()</span><br><span class="line"><span class="comment">#服务器设置在本地的cookie会保存在session中</span></span><br><span class="line">session.post(url,data,headers)</span><br><span class="line"><span class="comment">#会带上之前保存在session中的cookie</span></span><br><span class="line">sesion.get(url)</span><br></pre></td></tr></table></figure>

<p>例子</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">   <span class="keyword">import</span> requests</span><br><span class="line">   <span class="comment">#实例化session</span></span><br><span class="line">   session = requests.session()</span><br><span class="line">   <span class="comment">#表单的form的action的url地址</span></span><br><span class="line">post_url = <span class="string">""</span></span><br><span class="line">   headers = &#123;&#125;</span><br><span class="line">   post_data = &#123;<span class="string">"email"</span>:<span class="string">""</span>,<span class="string">"password"</span>:<span class="string">""</span>&#125;</span><br><span class="line">   session.post(post_url,header=headers,data=post_data)</span><br><span class="line">   <span class="comment"># 在使用session 请求登录后的页面</span></span><br><span class="line">   <span class="comment">#网页登录地址</span></span><br><span class="line">   url = <span class="string">""</span></span><br><span class="line">   response = session.get(url,headers=headers)</span><br><span class="line">   <span class="keyword">with</span> open(<span class="string">"文件"</span>,<span class="string">"w"</span>,encoding=<span class="string">"utf-8"</span>) <span class="keyword">as</span> f:</span><br><span class="line">       f.write(response.content.decode())</span><br></pre></td></tr></table></figure>

<p>解决：action url 地址在源代码中不可见的情况</p>
<p>Google控制台功能：选项Preserver log功能解决网页跳转时响应文件的丢失</p>
<p>可以找出login的action的url地址</p>
</li>
</ol>
<h6 id="数据提取方法值json"><a href="#数据提取方法值json" class="headerlink" title="数据提取方法值json"></a>数据提取方法值json</h6><p>json 是一种数据交换格式，像python类型（列表、字典）的字符串</p>
<p>json.loads</p>
<p>哪里会返回json的数据</p>
<p>​    流程器切换到手机版</p>
<p>​    抓包app</p>
<p>把json字符串转换为python类型：json.loads(json)字符串</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">str = input(<span class="string">"请输入中文："</span>)</span><br><span class="line">print(str)</span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line">url = <span class="string">"https://fanyi.baidu.com/basetrans"</span></span><br><span class="line"></span><br><span class="line">query_str = input(<span class="string">"输入要翻译的中文："</span>)</span><br><span class="line">data_string = &#123;</span><br><span class="line">   <span class="string">"query"</span>:query_str,</span><br><span class="line">   <span class="string">"from"</span>:<span class="string">"zh"</span>,</span><br><span class="line">   <span class="string">"to"</span>:<span class="string">"en"</span>&#125;</span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">"user-agent"</span>:</span><br><span class="line">    <span class="string">"Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"</span>,</span><br><span class="line">    <span class="string">"cookie"</span>:<span class="string">"值"</span></span><br><span class="line">    <span class="comment"># "referer":"https://fanyi.baidu.com/"</span></span><br><span class="line">&#125;</span><br><span class="line">response = requests.post(url,data=data_string,headers=headers)</span><br><span class="line">json_str = response.content.decode()</span><br><span class="line">jsons = json.loads(json_str)</span><br><span class="line">print(jsons)  </span><br><span class="line">print(type(jsons)) <span class="comment">#查看类型 字典类型</span></span><br><span class="line">ret = jsons[<span class="string">"trans"</span>][<span class="number">0</span>][<span class="string">"dst"</span>]</span><br><span class="line">print(<span class="string">"翻译结果："</span>,ret)</span><br></pre></td></tr></table></figure>

<p>把python类型转换为json字符串：json。dumps(“a:”a”,b:”b”)字符串</p>
<p>json.dumps({ })</p>
<p>json.dumps(ret1,ensure_ascii=False,indent=2)</p>
<p>​    ensure_ascil:让中文显示成中文</p>
<p>​    indent:能够让下一行在上一行的基础上空格</p>
<h6 id="数据提取方法值xpath和xml模块的学习"><a href="#数据提取方法值xpath和xml模块的学习" class="headerlink" title="数据提取方法值xpath和xml模块的学习"></a>数据提取方法值xpath和xml模块的学习</h6><p><strong>一门从html中提取数据的语言</strong></p>
<p>xpath语法:</p>
<p>xpath helper插件：帮助我们从elements中定位数据</p>
<p>使用</p>
<p> 选择节点</p>
<ol>
<li><p>/html/head/meta :能够选中html下的所有的meta标签</p>
</li>
<li><p>//：能够从任意节点开始</p>
</li>
<li><p>//li ：当前页面上的所有的li标签</p>
</li>
<li><p>/html/head//link:head下所有的link标签</p>
</li>
<li><p>@符号的用途  选择具体的某个元素</p>
</li>
<li><p>//div[@class=’class值’]/ul/li 选择  class=’’的div下的ul下的li标签</p>
<p>a/@href：选择a的href的值</p>
</li>
<li><p>获取文本  /a/text() text()方法  </p>
</li>
<li><p>获取a下的所有文本  /a//text()</p>
</li>
<li><p>./当前节点</p>
</li>
</ol>
<p>lxml </p>
<p>安装</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install lxml</span><br></pre></td></tr></table></figure>

<p>使用</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line">element = etree.HTML(<span class="string">"html字符串（ response.content.decode()）"</span>)</span><br><span class="line">element.xpath(<span class="string">"xpath地址"</span>)</span><br></pre></td></tr></table></figure>



<p>基础知识</p>
<ol>
<li>列表推导式</li>
</ol>
<ul>
<li>帮助我们快速的生成包含一堆数据的列表</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">[i+<span class="number">10</span> <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">10</span>)] </span><br><span class="line"><span class="comment"># ----&gt;[10,11,12,......,19]</span></span><br><span class="line"></span><br><span class="line">[<span class="string">"10月&#123;&#125;日"</span>.format(i) <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">1</span>,<span class="number">10</span>)] </span><br><span class="line"><span class="comment"># ------&gt;['"10月1日","10月2日"......."10月9日"]</span></span><br></pre></td></tr></table></figure>

<p>format:字符串格式化的一种方式</p>
<p>使用：”百度{}网盘”.format(“1”)</p>
<p>“百度{}网盘”.format([1,2,3,4])</p>
<p>“百度{}网盘{}”.format({1,2,3,4},[1,2,3,4])</p>
<p>“百度{}网盘{}”.format({1,2,3,4},1)</p>
<ol start="2">
<li>字典推导式</li>
</ol>
<ul>
<li>帮助我们快速生成包含一堆数据的字典</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">&#123;i+<span class="number">10</span>:i  <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">10</span>)&#125;</span><br><span class="line"><span class="comment">#&#123;10:0,11:1,.....19:9&#125;</span></span><br><span class="line">&#123;<span class="string">"a&#123;&#125;"</span>.format(i):<span class="number">10</span> <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">2</span>)&#125;</span><br><span class="line"><span class="comment">#&#123;"a0":10,"a1":10,"a2":10&#125;</span></span><br></pre></td></tr></table></figure>

<ol start="3">
<li>三元运算符</li>
</ol>
<ul>
<li>if 后面的条件成立，就把if前面的结果赋值给a,否则把else后面的结果赋值给a</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">a = <span class="number">10</span> <span class="keyword">if</span> <span class="number">4</span>&gt;<span class="number">3</span> <span class="keyword">else</span> <span class="number">20</span> 	<span class="comment">#a = 10</span></span><br><span class="line">a = <span class="number">10</span> <span class="keyword">if</span> <span class="number">4</span>&lt;<span class="number">3</span> <span class="keyword">else</span> <span class="number">20</span>   <span class="comment">#a = 20</span></span><br></pre></td></tr></table></figure>



<h6 id="写爬虫的讨论"><a href="#写爬虫的讨论" class="headerlink" title="写爬虫的讨论"></a>写爬虫的讨论</h6><ol>
<li><p>url（知道url地址的规律和总得页码数）:构造url地址地列表</p>
<p>start_url</p>
</li>
<li><p>发送请求，获取响应</p>
<p>requests</p>
</li>
<li><p>提取数据</p>
<p>返回json字符串:json模块</p>
<p>返回的是html字符串：lxml模块配合xpath提取数据</p>
</li>
<li><p>保存</p>
</li>
</ol>
<h6 id="Python中self用法"><a href="#Python中self用法" class="headerlink" title="Python中self用法"></a>Python中self用法</h6><p><a href="https://blog.csdn.net/CLHugh/article/details/75000104" target="_blank" rel="noopener">https://blog.csdn.net/CLHugh/article/details/75000104</a></p>
<p>self.xx是类里面的全局变量.</p>
<p>如果你知道java或者javascript中的this，那么self的作用和他们是一样的，不过我猜你可能也没接触过这两门语言。</p>
<h4 id="Beautiful-Soup-bs4-是解析、遍历、维护标签树的功能库-对应一个HTML-XML文档的全部内容"><a href="#Beautiful-Soup-bs4-是解析、遍历、维护标签树的功能库-对应一个HTML-XML文档的全部内容" class="headerlink" title="Beautiful Soup(bs4)    是解析、遍历、维护标签树的功能库,对应一个HTML/XML文档的全部内容"></a>Beautiful Soup(bs4)    是解析、遍历、维护标签树的功能库,对应一个HTML/XML文档的全部内容</h4><h6 id="概念-1"><a href="#概念-1" class="headerlink" title="概念"></a>概念</h6><ul>
<li>解析HTML页面信息标记与提取方法</li>
<li>解析、遍历、维护标签树的功能库,对应一个HTML/XML文档的全部内容</li>
</ul>
<h6 id="安装-1"><a href="#安装-1" class="headerlink" title="安装"></a>安装</h6><figure class="highlight cmd"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip install beautifulsoup4</span><br></pre></td></tr></table></figure>

<h6 id="使用"><a href="#使用" class="headerlink" title="使用"></a>使用</h6><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 引用方式：</span></span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup||<span class="keyword">import</span> bs4</span><br><span class="line">soup = BeautifulSoup(<span class="string">"&lt;html&gt;data&lt;/html&gt;"</span>,<span class="string">'html.parser'</span>)</span><br><span class="line">soup = BeautifulSoup(open(<span class="string">"D://demo.html"</span>),<span class="string">'html.parser'</span>)</span><br><span class="line">soup.prettify()</span><br></pre></td></tr></table></figure>

<h6 id="基本元素"><a href="#基本元素" class="headerlink" title="基本元素"></a>基本元素</h6><h6 id="遍历方法"><a href="#遍历方法" class="headerlink" title="遍历方法"></a>遍历方法</h6><p><strong>上行遍历</strong></p>
<ul>
<li>.parent 父亲标签</li>
<li>.parents  迭代类型 先辈</li>
</ul>
<p><strong>下行遍历</strong></p>
<p><strong>平行遍历</strong></p>
<ul>
<li>.next_sibling 文本顺序的平行节点的下一个节点</li>
<li>.previous.sibling 文本顺序的平行节点的上一个节点</li>
<li>.next_siblings 迭代类型，文本顺序的平行节点的遍历后续节点</li>
<li>.previous.siblings 迭代类型，文本顺序的平行节点的遍历前续节点</li>
</ul>
<p><strong>扩展方法</strong></p>
<p><strong>简便写法</strong></p>
<p><strong>信息的标记</strong></p>
<p><strong>信息的提取</strong></p>
<h6 id="解析器"><a href="#解析器" class="headerlink" title="解析器"></a>解析器</h6><ul>
<li><p>安装</p>
<figure class="highlight cmd"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">pip install lxml</span><br><span class="line">pip install html5lib</span><br></pre></td></tr></table></figure>
</li>
<li><p>分类</p>
<p><strong>bs4的HTML解析器</strong></p>
<p>BeautifulSoup(mk,’html.parser’)</p>
<p><strong>lxml的HTML解析器</strong></p>
<p>BeautifulSoup(mk,’lxml’) </p>
<p><strong>lxml的XML解析器</strong></p>
<p>BeautifulSoup(mk,,’xml’)</p>
<p><strong>html5lib的HTML解析器</strong> </p>
<p>BeautifulSoup(mk,’html5lib’)</p>
</li>
</ul>
<h4 id="robots-txt-网络爬虫排除标准"><a href="#robots-txt-网络爬虫排除标准" class="headerlink" title="robots.txt(网络爬虫排除标准)"></a>robots.txt(网络爬虫排除标准)</h4><h6 id="robots-txt"><a href="#robots-txt" class="headerlink" title="robots.txt"></a>robots.txt</h6><ul>
<li>网络爬虫排除标准</li>
</ul>
<h6 id="来源审查"><a href="#来源审查" class="headerlink" title="来源审查"></a>来源审查</h6><ul>
<li>判断User-Agent进行限制</li>
<li>检查来访HTTP协议头的User-Agent域，响应浏览器或友好爬虫的访问</li>
</ul>
<h6 id="分布公告"><a href="#分布公告" class="headerlink" title="分布公告"></a>分布公告</h6><ul>
<li>Robots协议<br>Rebot Exclusion Standard<br>网络爬虫排除标准</li>
<li>robots.txt文件<br>告知所有爬虫网站的爬取策略</li>
</ul>
<h3 id="Scrapy（中级）"><a href="#Scrapy（中级）" class="headerlink" title="Scrapy（中级）"></a>Scrapy（中级）</h3><h2 id="实例"><a href="#实例" class="headerlink" title="实例"></a>实例</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line">r = requests.get(<span class="string">"https://www.baidu.com/"</span>)</span><br><span class="line">print(r.status_code)     <span class="comment">#获取网页状态码</span></span><br><span class="line">print(r.url)             <span class="comment">#获取网址</span></span><br><span class="line">print(r.text)            <span class="comment">#获取网页源代码</span></span><br></pre></td></tr></table></figure>

<p>IDE安装</p>
<p>爬取源代码</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line">url = <span class="string">"http://www.baidu.com"</span></span><br><span class="line">respones = requests.get(url)</span><br><span class="line">print(respones.content.encode())</span><br></pre></td></tr></table></figure>

<p>爬取指定文字</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># coding=utf-8</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> urllib2</span><br><span class="line"><span class="keyword">import</span> urllib</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> thread</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line"></span><br><span class="line">url=<span class="string">"http://toutiao.sogou.com/?fr=qqxwtt"</span></span><br><span class="line">page = urllib.urlopen(url)<span class="comment">#打开网址</span></span><br><span class="line">html = page.read()        <span class="comment">#读取网页内容，保存到htlm中</span></span><br><span class="line">bs0bj=BeautifulSoup(html) <span class="comment">#创建一个beautifulsoup的类</span></span><br><span class="line">namelist=bs0bj.findAll(<span class="string">"a"</span>)<span class="comment">#通过标签筛选文字信息</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> name <span class="keyword">in</span> namelist:</span><br><span class="line">    <span class="keyword">print</span> (name.get_text())</span><br></pre></td></tr></table></figure>

<p>爬取单张图片</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 爬取图片</span></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line">root = <span class="string">"D:/Downloads/"</span></span><br><span class="line">url = <span class="string">"http://b-ssl.duitang.com/uploads/item/201305/26/20130526140022_5fMJe.jpeg"</span></span><br><span class="line">path = root +url.split(<span class="string">'/'</span>)[<span class="number">-1</span>]</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(root):</span><br><span class="line">        os.mkdir(root)</span><br><span class="line">    <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(path):</span><br><span class="line">        r = requests.get(url)</span><br><span class="line">        <span class="keyword">with</span> open(path,<span class="string">'wb'</span>) <span class="keyword">as</span> f:</span><br><span class="line">            f.write(r.content)</span><br><span class="line">            f.close()</span><br><span class="line">            print(<span class="string">"成功"</span>)</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        print(<span class="string">"已保存"</span>)</span><br><span class="line"><span class="keyword">except</span>:</span><br><span class="line">    print(<span class="string">"失败"</span>)</span><br></pre></td></tr></table></figure>

<p>爬取多张图片</p>
<p>爬取一个视频</p>
<p>爬取多个视频</p>
<p>百度翻译</p>
<p>（python3）</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># coding: utf8</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line"><span class="string">    @Author: LCY</span></span><br><span class="line"><span class="string">    @Contact: lchuanyong@126.com</span></span><br><span class="line"><span class="string">    @blog: http://http://blog.csdn.net/lcyong_</span></span><br><span class="line"><span class="string">    @Date: 2018-01-15</span></span><br><span class="line"><span class="string">    @Time: 19:19</span></span><br><span class="line"><span class="string">    说明： appid和secretKey为百度翻译文档中自带的，需要切换为自己的</span></span><br><span class="line"><span class="string">           python2和python3部分库名称更改对应如下：</span></span><br><span class="line"><span class="string">           httplib      ----&gt;    http.client</span></span><br><span class="line"><span class="string">           md5          ----&gt;    hashlib.md5</span></span><br><span class="line"><span class="string">           urllib.quote ----&gt;    urllib.parse.quote</span></span><br><span class="line"><span class="string">    官方链接：</span></span><br><span class="line"><span class="string">           http://api.fanyi.baidu.com/api/trans/product/index</span></span><br><span class="line"><span class="string">           </span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line"> </span><br><span class="line"><span class="keyword">import</span> http.client</span><br><span class="line"><span class="keyword">import</span> hashlib</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> urllib</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"> </span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">baidu_translate</span><span class="params">(content)</span>:</span></span><br><span class="line">    <span class="comment">#appid api id 密钥 我的翻译地址myurl</span></span><br><span class="line">    appid = <span class="string">'20151113000005349'</span></span><br><span class="line">    secretKey = <span class="string">'osubCEzlGjzvw8qdQc41'</span></span><br><span class="line">    httpClient = <span class="literal">None</span></span><br><span class="line">    myurl = <span class="string">'/api/trans/vip/translate'</span></span><br><span class="line">    </span><br><span class="line">    q = content</span><br><span class="line">    fromLang = <span class="string">'zh'</span> <span class="comment"># 源语言</span></span><br><span class="line">    toLang = <span class="string">'en'</span>   <span class="comment"># 翻译后的语言</span></span><br><span class="line">    salt = random.randint(<span class="number">32768</span>, <span class="number">65536</span>)</span><br><span class="line">    sign = appid + q + str(salt) + secretKey</span><br><span class="line">    sign = hashlib.md5(sign.encode()).hexdigest()</span><br><span class="line">    myurl = myurl + <span class="string">'?appid='</span> + appid + <span class="string">'&amp;q='</span> + urllib.parse.quote(</span><br><span class="line">        q) + <span class="string">'&amp;from='</span> + fromLang + <span class="string">'&amp;to='</span> + toLang + <span class="string">'&amp;salt='</span> + str(</span><br><span class="line">        salt) + <span class="string">'&amp;sign='</span> + sign</span><br><span class="line"> </span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        httpClient = http.client.HTTPConnection(<span class="string">'api.fanyi.baidu.com'</span>)</span><br><span class="line">        httpClient.request(<span class="string">'GET'</span>, myurl)</span><br><span class="line">        <span class="comment"># response是HTTPResponse对象</span></span><br><span class="line">        response = httpClient.getresponse()</span><br><span class="line">        jsonResponse = response.read().decode(<span class="string">"utf-8"</span>)<span class="comment"># 获得返回的结果，结果为json格式</span></span><br><span class="line">        js = json.loads(jsonResponse)  <span class="comment"># 将json格式的结果转换字典结构</span></span><br><span class="line">        dst = str(js[<span class="string">"trans_result"</span>][<span class="number">0</span>][<span class="string">"dst"</span>])  <span class="comment"># 取得翻译后的文本结果</span></span><br><span class="line">        print(dst) <span class="comment"># 打印结果</span></span><br><span class="line">    <span class="keyword">except</span> Exception <span class="keyword">as</span> e:</span><br><span class="line">        print(e)</span><br><span class="line">    <span class="keyword">finally</span>:</span><br><span class="line">        <span class="keyword">if</span> httpClient:</span><br><span class="line">            httpClient.close()</span><br><span class="line"> </span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    <span class="keyword">while</span> <span class="literal">True</span>:  <span class="comment">#死循环</span></span><br><span class="line">        print(<span class="string">"请输入要翻译的内容,如果退出输入q"</span>)</span><br><span class="line">        content = input()</span><br><span class="line">        <span class="keyword">if</span> (content == <span class="string">'q'</span>):</span><br><span class="line">            <span class="keyword">break</span>  <span class="comment">#q退出</span></span><br><span class="line">        baidu_translate(content)</span><br></pre></td></tr></table></figure>

<p>官方（python2）</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#/usr/bin/env python</span></span><br><span class="line"><span class="comment">#coding=utf8</span></span><br><span class="line"> </span><br><span class="line"><span class="keyword">import</span> httplib</span><br><span class="line"><span class="keyword">import</span> md5</span><br><span class="line"><span class="keyword">import</span> urllib</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"> </span><br><span class="line">appid = <span class="string">'20151113000005349'</span></span><br><span class="line">secretKey = <span class="string">'osubCEzlGjzvw8qdQc41'</span></span><br><span class="line"> </span><br><span class="line"> </span><br><span class="line">httpClient = <span class="literal">None</span></span><br><span class="line">myurl = <span class="string">'/api/trans/vip/translate'</span></span><br><span class="line">q = <span class="string">'apple'</span></span><br><span class="line">fromLang = <span class="string">'en'</span></span><br><span class="line">toLang = <span class="string">'zh'</span></span><br><span class="line">salt = random.randint(<span class="number">32768</span>, <span class="number">65536</span>)</span><br><span class="line"> </span><br><span class="line">sign = appid+q+str(salt)+secretKey</span><br><span class="line">m1 = md5.new()</span><br><span class="line">m1.update(sign)</span><br><span class="line">sign = m1.hexdigest()</span><br><span class="line">myurl = myurl+<span class="string">'?appid='</span>+appid+<span class="string">'&amp;q='</span>+urllib.quote(q)+<span class="string">'&amp;from='</span>+fromLang+<span class="string">'&amp;to='</span>+toLang+<span class="string">'&amp;salt='</span>+str(salt)+<span class="string">'&amp;sign='</span>+sign</span><br><span class="line"> </span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    httpClient = httplib.HTTPConnection(<span class="string">'api.fanyi.baidu.com'</span>)</span><br><span class="line">    httpClient.request(<span class="string">'GET'</span>, myurl)</span><br><span class="line"> </span><br><span class="line">    <span class="comment">#response是HTTPResponse对象</span></span><br><span class="line">    response = httpClient.getresponse()</span><br><span class="line">    <span class="keyword">print</span> response.read()</span><br><span class="line"><span class="keyword">except</span> Exception, e:</span><br><span class="line">    <span class="keyword">print</span> e</span><br><span class="line"><span class="keyword">finally</span>:</span><br><span class="line">    <span class="keyword">if</span> httpClient:</span><br><span class="line">        httpClient.close()</span><br></pre></td></tr></table></figure>

<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">BiliBiliSpider</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line">        self.url_temp =<span class="string">"https://www.bilibili.com/v/dance/otaku/#/all/default/0/&#123;&#125;/"</span></span><br><span class="line">        self.headers = &#123;</span><br><span class="line">            <span class="string">"User-Agent"</span>:</span><br><span class="line">                <span class="string">"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)     Chrome/80.0.3987.106 Safari/537.36"</span></span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 1.根据url地址的规律，构造url list</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get_url_list</span><span class="params">(self)</span>:</span></span><br><span class="line">        url_list = [self.url_temp.format(i) <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">1</span>,<span class="number">10</span>)]</span><br><span class="line">        <span class="keyword">return</span> url_list</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 2.发送请求，获取响应</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse_url</span><span class="params">(self,url)</span>:</span></span><br><span class="line">        response = requests.get(url,headers=self.headers)</span><br><span class="line">        <span class="keyword">return</span>  response.content.decode()</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 3.提取数据</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">get_content_list</span><span class="params">(self,html_str)</span>:</span></span><br><span class="line">        html = etree.HTML(html_str)</span><br><span class="line">        <span class="comment">#1.分组</span></span><br><span class="line">        div_list = html.xpath(<span class="string">"//*[@class=vd-list-cnt]"</span>)</span><br><span class="line">        print(div_list)</span><br><span class="line">        content_list = []</span><br><span class="line">        <span class="keyword">for</span> div <span class="keyword">in</span> div_list:</span><br><span class="line">            item = []</span><br><span class="line">            <span class="comment"># //*[@id="videolist_box"]/div[2]</span></span><br><span class="line">            item[<span class="string">""</span>] = div.xpath(<span class="string">".//ul/li[1]/div/div[2]/a[@title]"</span>)</span><br><span class="line">            content_list.append(item)</span><br><span class="line">        print(content_list)</span><br><span class="line">        <span class="keyword">return</span>  content_list</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 4.保存</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">save_content_list</span><span class="params">(self,content_list)</span>:</span></span><br><span class="line">        <span class="keyword">with</span> open(<span class="string">"bilibili.txt"</span>,<span class="string">"w"</span>,encoding=<span class="string">"UTF-8"</span>) <span class="keyword">as</span> f:</span><br><span class="line">            <span class="keyword">for</span> content <span class="keyword">in</span> content_list:</span><br><span class="line">                f.write(json.dumps(content,ensure_ascii=<span class="literal">False</span>))</span><br><span class="line">                f.write(<span class="string">"\n"</span>)</span><br><span class="line"></span><br><span class="line">        print(<span class="string">"保存成功"</span>)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">run</span><span class="params">(self)</span>:</span><span class="comment">#实现主要逻辑</span></span><br><span class="line">        <span class="comment">#1.根据url地址的规律，构造url list</span></span><br><span class="line">        url_list = self.get_url_list()</span><br><span class="line">        <span class="comment">#2.发送请求，获取响应</span></span><br><span class="line">        <span class="keyword">for</span> url <span class="keyword">in</span> url_list:</span><br><span class="line"></span><br><span class="line">            html_str = self.parse_url(url)</span><br><span class="line">        <span class="comment"># 3.提取数据</span></span><br><span class="line">            content_list = self.get_content_list(html_str)</span><br><span class="line">        <span class="comment">#4.保存</span></span><br><span class="line">            print(url)</span><br><span class="line">            self.save_content_list(content_list)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 主函数</span></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">   bilibili = BiliBiliSpider()</span><br><span class="line">   bilibili.run()</span><br></pre></td></tr></table></figure>

<p><a href="https://zhuanlan.zhihu.com/p/73742321" target="_blank" rel="noopener">https://zhuanlan.zhihu.com/p/73742321</a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br></pre></td><td class="code"><pre><span class="line">import threading</span><br><span class="line">import time</span><br><span class="line">import requests</span><br><span class="line">from bs4 import BeautifulSoup</span><br><span class="line">import os</span><br><span class="line">import random</span><br><span class="line"></span><br><span class="line">url &#x3D; &quot;http:&#x2F;&#x2F;www.mzitu.com&#x2F;all&quot;</span><br><span class="line">&#39;&#39;&#39; 获取html页面&#39;&#39;&#39;</span><br><span class="line"></span><br><span class="line">def downHtml(url):</span><br><span class="line">    headers &#x3D; &#123;</span><br><span class="line">        &#39;User-Agent&#39;: &quot;Mozilla&#x2F;5.0 (Windows NT 10.0; Win64; x64) AppleWebKit&#x2F;537.36 (KHTML, like Gecko) Chrome&#x2F;42.0.2311.135 Safari&#x2F;537.36 Edge&#x2F;12.10240&quot;,</span><br><span class="line">        &#39;Connection&#39;: &#39;Keep-Alive&#39;,</span><br><span class="line">        &#39;Referer&#39;: &quot;http:&#x2F;&#x2F;www.mzitu.com&#x2F;99566&quot;</span><br><span class="line">    &#125;</span><br><span class="line">    return requests.get(url, headers&#x3D;headers).text</span><br><span class="line"></span><br><span class="line">&#39;&#39;&#39; 解析获取所有的妹子url  &#39;&#39;&#39;</span><br><span class="line"></span><br><span class="line">def parseHtml(html):</span><br><span class="line">    soup &#x3D; BeautifulSoup(html, &quot;lxml&quot;)</span><br><span class="line">    list &#x3D; soup.select(&quot;a[target&#x3D;&#39;_blank&#39;]&quot;)</span><br><span class="line">    mylist &#x3D; [];</span><br><span class="line">    count &#x3D; 0  # 添加一个统计链接的数量</span><br><span class="line">    for a in list:</span><br><span class="line">        # print(a[&quot;href&quot;], a.get_text())</span><br><span class="line">        # print(a[&quot;href&quot;])</span><br><span class="line">        mylist.append(a[&quot;href&quot;])</span><br><span class="line">    return mylist</span><br><span class="line">    # print(count)</span><br><span class="line">    #     print(mylist)</span><br><span class="line">    # print(len(mylist))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">&#39;&#39;&#39; 获取妹子所有图片的数量 &#39;&#39;&#39;</span><br><span class="line"></span><br><span class="line">def get_pic_num(html):</span><br><span class="line">    soup &#x3D; BeautifulSoup(html, &quot;lxml&quot;)</span><br><span class="line">    list &#x3D; soup.find(&quot;div&quot;, class_&#x3D;&quot;pagenavi&quot;).find_all(&quot;span&quot;)</span><br><span class="line">    # print(list)</span><br><span class="line">    # list [&lt;span&gt;«上一组&lt;&#x2F;span&gt;, &lt;span&gt;1&lt;&#x2F;span&gt;, &lt;span&gt;2&lt;&#x2F;span&gt;, &lt;span&gt;3&lt;&#x2F;span&gt;, &lt;span&gt;4&lt;&#x2F;span&gt;, &lt;span class&#x3D;&quot;dots&quot;&gt;…&lt;&#x2F;span&gt;, &lt;span&gt;65&lt;&#x2F;span&gt;, &lt;span&gt;下一页»&lt;&#x2F;span&gt;]</span><br><span class="line">    return int(list[-2].string)  # ？</span><br><span class="line"></span><br><span class="line">&#39;&#39;&#39; 获取妹子的图片 &#39;&#39;&#39;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"># 获取所有妹子的url</span><br><span class="line">def get_pic(html):</span><br><span class="line">    soup &#x3D; BeautifulSoup(html, &quot;lxml&quot;)</span><br><span class="line">    img &#x3D; soup.find(&quot;div&quot;, class_&#x3D;&quot;main-image&quot;).find_all(&quot;img&quot;)</span><br><span class="line">    print(img[0][&quot;src&quot;], img[0][&quot;alt&quot;])</span><br><span class="line">    download_pic(img[0][&quot;src&quot;], img[0][&quot;alt&quot;])</span><br><span class="line">    </span><br><span class="line">&#39;&#39;&#39; 下载妹子的图片 &#39;&#39;&#39;</span><br><span class="line">def download_pic(url, himg):</span><br><span class="line">    headers &#x3D; &#123;</span><br><span class="line">        &#39;User-Agent&#39;: &quot;Mozilla&#x2F;5.0 (Windows NT 10.0; Win64; x64) AppleWebKit&#x2F;537.36 (KHTML, like Gecko) Chrome&#x2F;42.0.2311.135 Safari&#x2F;537.36 Edge&#x2F;12.10240&quot;,</span><br><span class="line">        &#39;Connection&#39;: &#39;Keep-Alive&#39;,</span><br><span class="line">        &#39;Referer&#39;: &quot;http:&#x2F;&#x2F;www.mzitu.com&#x2F;99566&quot;</span><br><span class="line">    &#125;</span><br><span class="line">    img &#x3D; requests.get(url, headers&#x3D;headers)</span><br><span class="line"></span><br><span class="line">    t &#x3D; time.time()</span><br><span class="line">    nowTime &#x3D; lambda: int(round(t * 1000))</span><br><span class="line"></span><br><span class="line">    # soup &#x3D; BeautifulSoup(html, &quot;lxml&quot;)</span><br><span class="line">    # img &#x3D; soup.find(&quot;div&quot;, class_&#x3D;&quot;main-image&quot;).find_all(&quot;img&quot;)</span><br><span class="line">    # pname &#x3D; img[0][&quot;src&quot;]</span><br><span class="line">    print(himg)</span><br><span class="line">    folder &#x3D; os.path.exists(&quot;H:\mztu\\&quot; + himg)</span><br><span class="line">    if not folder:  # 判断是否存在文件夹如果不存在则创建为文件夹</span><br><span class="line">        os.makedirs(&quot;H:\mztu\\&quot; + himg)</span><br><span class="line">    with open(&quot;H:\mztu\&#123;&#125;\&#123;&#125;.jpg&quot;.format(himg, nowTime()), &#39;ab&#39;) as f:</span><br><span class="line">        f.write(img.content)</span><br><span class="line"></span><br><span class="line"># 创建文件夹</span><br><span class="line">html &#x3D; downHtml(url)</span><br><span class="line">list &#x3D; parseHtml(html)</span><br><span class="line"># print(list)</span><br><span class="line"></span><br><span class="line">for url in list:</span><br><span class="line">    # print(src)</span><br><span class="line">    # 获取妹子所有的图片</span><br><span class="line">    html &#x3D; downHtml(url)</span><br><span class="line">    num &#x3D; get_pic_num(html)</span><br><span class="line">    print(num)</span><br><span class="line">    for i in range(num):</span><br><span class="line">        str &#x3D; &quot;&#123;&#125;&#123;&#125;&#123;&#125;&quot;.format(url, &quot;&#x2F;&quot;, i + 1)</span><br><span class="line">        html &#x3D; downHtml(str)</span><br><span class="line">        t &#x3D; threading.Thread(target&#x3D;get_pic, args&#x3D;(html,))</span><br><span class="line">        t.start()</span><br><span class="line">        t.join()</span><br></pre></td></tr></table></figure>


    </div>

    
    
    
        <div class="reward-container">
  <div></div>
  <button onclick="var qr = document.getElementById('qr'); qr.style.display = (qr.style.display === 'none') ? 'block' : 'none';">
    打赏
  </button>
  <div id="qr" style="display: none;">
      
      <div style="display: inline-block;">
        <img src="/images/wechatpay.jpg" alt="heweiliang 微信支付">
        <p>微信支付</p>
      </div>
      
      <div style="display: inline-block;">
        <img src="/images/alipay.jpg" alt="heweiliang 支付宝">
        <p>支付宝</p>
      </div>

  </div>
</div>

        

  <div class="followme">
    <p>欢迎关注我的其它发布渠道</p>

    <div class="social-list">

        <div class="social-item">
          <a target="_blank" class="social-link" href="/atom.xml">
            <span class="icon">
              <i class="fa fa-rss"></i>
            </span>

            <span class="label">RSS</span>
          </a>
        </div>
    </div>
  </div>


      <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/Python/" rel="tag"># Python</a>
              <a href="/tags/%E7%88%AC%E8%99%AB/" rel="tag"># 爬虫</a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item">
    <a href="/2020/02/23/%E8%AE%A1%E7%AE%97%E6%9C%BA%E5%9F%BA%E7%A1%80/%E8%AE%A1%E7%AE%97%E6%9C%BA%E7%BB%84%E8%A3%85%E4%B8%8E%E7%BB%B4%E6%8A%A4/" rel="prev" title="计算机基础/计算机组装与维护">
      <i class="fa fa-chevron-left"></i> 计算机基础/计算机组装与维护
    </a></div>
      <div class="post-nav-item">
    <a href="/2020/02/27/%E5%89%8D%E7%AB%AF/%E7%BD%91%E9%A1%B5%E3%80%81APP%E3%80%81%E5%B0%8F%E7%A8%8B%E5%BA%8F%E3%80%81PC/WeChatApplet/%E5%BE%AE%E4%BF%A1%E5%B0%8F%E7%A8%8B%E5%BA%8F/" rel="next" title="前端/网页、APP、小程序、PC/WeChatApplet/微信小程序">
      前端/网页、APP、小程序、PC/WeChatApplet/微信小程序 <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          
    
  <div class="comments">
    <div id="lv-container" data-id="city" data-uid="MTAyMC81MDY1MC8yNzEzMw=="></div>
  </div>
  

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#python爬虫-（2020-2-25-2-28）"><span class="nav-number">1.</span> <span class="nav-text">python爬虫 （2020.2.25-2.28）</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#基础知识"><span class="nav-number">1.0.1.</span> <span class="nav-text">基础知识</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Requests-Beautiful-Soup-robots-txt-初级"><span class="nav-number">1.0.2.</span> <span class="nav-text">Requests + Beautiful Soup + robots.txt(初级)</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Requests-爬取HTML页面自动网络请求提交"><span class="nav-number">1.0.3.</span> <span class="nav-text">Requests    爬取HTML页面自动网络请求提交</span></a><ol class="nav-child"><li class="nav-item nav-level-6"><a class="nav-link" href="#概念"><span class="nav-number">1.0.3.0.1.</span> <span class="nav-text">概念</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#安装"><span class="nav-number">1.0.3.0.2.</span> <span class="nav-text">安装</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#requests方法"><span class="nav-number">1.0.3.0.3.</span> <span class="nav-text">requests方法</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#response对象的属性"><span class="nav-number">1.0.3.0.4.</span> <span class="nav-text">response对象的属性</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#发送带header的请求-post-get"><span class="nav-number">1.0.3.0.5.</span> <span class="nav-text">发送带header的请求 post&#x2F;get</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#搜索引擎关键词提交接口"><span class="nav-number">1.0.3.0.6.</span> <span class="nav-text">搜索引擎关键词提交接口</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#Retrying模块（解决超时请求问题的库）"><span class="nav-number">1.0.3.0.7.</span> <span class="nav-text">Retrying模块（解决超时请求问题的库）</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#Cookie的相关请求"><span class="nav-number">1.0.3.0.8.</span> <span class="nav-text">Cookie的相关请求</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#数据提取方法值json"><span class="nav-number">1.0.3.0.9.</span> <span class="nav-text">数据提取方法值json</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#数据提取方法值xpath和xml模块的学习"><span class="nav-number">1.0.3.0.10.</span> <span class="nav-text">数据提取方法值xpath和xml模块的学习</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#写爬虫的讨论"><span class="nav-number">1.0.3.0.11.</span> <span class="nav-text">写爬虫的讨论</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#Python中self用法"><span class="nav-number">1.0.3.0.12.</span> <span class="nav-text">Python中self用法</span></a></li></ol></li></ol></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Beautiful-Soup-bs4-是解析、遍历、维护标签树的功能库-对应一个HTML-XML文档的全部内容"><span class="nav-number">1.0.4.</span> <span class="nav-text">Beautiful Soup(bs4)    是解析、遍历、维护标签树的功能库,对应一个HTML&#x2F;XML文档的全部内容</span></a><ol class="nav-child"><li class="nav-item nav-level-6"><a class="nav-link" href="#概念-1"><span class="nav-number">1.0.4.0.1.</span> <span class="nav-text">概念</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#安装-1"><span class="nav-number">1.0.4.0.2.</span> <span class="nav-text">安装</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#使用"><span class="nav-number">1.0.4.0.3.</span> <span class="nav-text">使用</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#基本元素"><span class="nav-number">1.0.4.0.4.</span> <span class="nav-text">基本元素</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#遍历方法"><span class="nav-number">1.0.4.0.5.</span> <span class="nav-text">遍历方法</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#解析器"><span class="nav-number">1.0.4.0.6.</span> <span class="nav-text">解析器</span></a></li></ol></li></ol></li><li class="nav-item nav-level-4"><a class="nav-link" href="#robots-txt-网络爬虫排除标准"><span class="nav-number">1.0.5.</span> <span class="nav-text">robots.txt(网络爬虫排除标准)</span></a><ol class="nav-child"><li class="nav-item nav-level-6"><a class="nav-link" href="#robots-txt"><span class="nav-number">1.0.5.0.1.</span> <span class="nav-text">robots.txt</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#来源审查"><span class="nav-number">1.0.5.0.2.</span> <span class="nav-text">来源审查</span></a></li><li class="nav-item nav-level-6"><a class="nav-link" href="#分布公告"><span class="nav-number">1.0.5.0.3.</span> <span class="nav-text">分布公告</span></a></li></ol></li></ol></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#Scrapy（中级）"><span class="nav-number">1.1.</span> <span class="nav-text">Scrapy（中级）</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#实例"><span class="nav-number">2.</span> <span class="nav-text">实例</span></a></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image" alt="heweiliang"
      src="/images/avatar.jpg">
  <p class="site-author-name" itemprop="name">heweiliang</p>
  <div class="site-description" itemprop="description">heweiliang blog</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">96</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">18</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">24</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/yourname" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;yourname" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i></a>
      </span>
      <span class="links-of-author-item">
        <a href="mailto:yourname@gmail.com" title="E-Mail → mailto:yourname@gmail.com" rel="noopener" target="_blank"><i class="fa fa-fw fa-envelope"></i></a>
      </span>
      <span class="links-of-author-item">
        <a href="https://plus.google.com/yourname" title="Google → https:&#x2F;&#x2F;plus.google.com&#x2F;yourname" rel="noopener" target="_blank"><i class="fa fa-fw fa-google"></i></a>
      </span>
      <span class="links-of-author-item">
        <a href="https://stackoverflow.com/yourname" title="StackOverflow → https:&#x2F;&#x2F;stackoverflow.com&#x2F;yourname" rel="noopener" target="_blank"><i class="fa fa-fw fa-stack-overflow"></i></a>
      </span>
  </div>
  <div class="cc-license motion-element" itemprop="license">
    <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" class="cc-opacity" rel="noopener" target="_blank"><img src="/images/cc-by-nc-sa.svg" alt="Creative Commons"></a>
  </div>


  <div class="links-of-blogroll motion-element">
    <div class="links-of-blogroll-title">
      <i class="fa fa-fw fa-link"></i>
      Links
    </div>
    <ul class="links-of-blogroll-list">
        <li class="links-of-blogroll-item">
          <a href="http://yoursite.com/" title="http:&#x2F;&#x2F;yoursite.com">Title</a>
        </li>
        <li class="links-of-blogroll-item">
          <a href="http://yoursite.com/" title="http:&#x2F;&#x2F;yoursite.com">Title2</a>
        </li>
        <li class="links-of-blogroll-item">
          <a href="http://yoursite.com/" title="http:&#x2F;&#x2F;yoursite.com">Title3</a>
        </li>
    </ul>
  </div>

      </div>
        <div class="back-to-top motion-element">
          <i class="fa fa-arrow-up"></i>
          <span>0%</span>
        </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        

<div class="copyright">
  
  &copy; 
  <span itemprop="copyrightYear">2020</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">heweiliang</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-area-chart"></i>
    </span>
    <span title="站点总字数">376k</span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item-icon">
      <i class="fa fa-coffee"></i>
    </span>
    <span title="站点阅读时长">5:42</span>
</div>

        
<div class="busuanzi-count">
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    <span class="post-meta-item" id="busuanzi_container_site_uv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-user"></i>
      </span>
      <span class="site-uv" title="总访客量">
        <span id="busuanzi_value_site_uv"></span>
      </span>
    </span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item" id="busuanzi_container_site_pv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-eye"></i>
      </span>
      <span class="site-pv" title="总访问量">
        <span id="busuanzi_value_site_pv"></span>
      </span>
    </span>
</div>








      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/pisces.js"></script>


<script src="/js/next-boot.js"></script>

<script src="/js/bookmark.js"></script>


  <script defer src="/lib/three/three.min.js"></script>
    <script defer src="/lib/three/three-waves.min.js"></script>
    <script defer src="/lib/three/canvas_lines.min.js"></script>


  




  
<script src="/js/local-search.js"></script>













  

  

<script>
NexT.utils.loadComments(document.querySelector('#lv-container'), () => {
  window.livereOptions = {
    refer: location.pathname.replace(CONFIG.root, '').replace('index.html', '')
  };
  (function(d, s) {
    var j, e = d.getElementsByTagName(s)[0];
    if (typeof LivereTower === 'function') { return; }
    j = d.createElement(s);
    j.src = 'https://cdn-city.livere.com/js/embed.dist.js';
    j.async = true;
    e.parentNode.insertBefore(j, e);
  })(document, 'script');
});
</script>

<script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"pluginModelPath":"assets/","model":{"jsonPath":"/live2dw/assets/Epsilon2.1.model.json"},"display":{"position":"left","width":300,"height":600},"mobile":{"show":false},"log":false,"pluginJsPath":"lib/","pluginRootPath":"live2dw/","tagMode":false});</script></body>
</html>
